#!/usr/bin/env python
# -*- coding:utf-8 -*-
# @Time      :2021/3/26 14:23
# @Author    :cjw
import requests
from random import uniform
import time
from lxml import etree
import csv
import psutil
from concurrent.futures import ThreadPoolExecutor

f = open('./菜价价格行情.csv', 'w', encoding='utf-8', newline='')
csv_write = csv.writer(f)
headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.90 Safari/537.36'
}


def download_one_page(url: str):
    """爬取北京新发地单页面数据"""
    # 随机请求等待时间
    sleep_time = round(uniform(0.1, 1), 2)
    time.sleep(sleep_time)

    resp = requests.get(url, headers=headers)
    html = etree.HTML(resp.text)
    table = html.xpath('//table[@class="hq_table"]')[0]
    trs = table.xpath('.//tr[position()>1]')

    page_text = []  # 当前页面数据
    for tr in trs:
        tds_txt = tr.xpath('.//td/text()')
        tds_txt = (txt.replace('\\', '').replace('/', '') for txt in tds_txt)
        page_text.append(list(tds_txt))
    csv_write.writerows(page_text)
    resp.close()
    print(url, f'提取完毕,数据数量: {len(page_text)}', f'随机等待时间：{sleep_time}s')


if __name__ == '__main__':
    # 创建线程池
    thread_nums = psutil.cpu_count(False) * 2 + 1
    t1 = time.time()
    with ThreadPoolExecutor(50) as t:
        for i in range(1, 15085):
            url = f'http://www.xinfadi.com.cn/marketanalysis/0/list/{i}.shtml'
            # 把下载任务提交给线程池
            t.submit(download_one_page, url)
    t2 = time.time()
    print(f'下载耗费时间：{round((t2 - t1), 2)}')
    print('全部下载完毕！')
    f.close()
